{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Benchmarking Behavior Planners in BARK\n",
    "\n",
    "This notebook the benchmarking workflow of BARK.\n",
    "\n",
    "Systematically benchmarking behavior consists of\n",
    "1. A reproducable set of scenarios (we call it **BenchmarkDatabase**)\n",
    "2. Metrics, which you use to study the performance (we call it **Evaluators**)\n",
    "3. The behavior model(s) under test\n",
    "\n",
    "Our **BenchmarkRunner** can then run the benchmark and produce the results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython.display import Video\n",
    "\n",
    "from benchmark_database.load.benchmark_database import BenchmarkDatabase\n",
    "from benchmark_database.serialization.database_serializer import DatabaseSerializer\n",
    "from bark.benchmark.benchmark_runner import BenchmarkRunner, BenchmarkConfig, BenchmarkResult\n",
    "from bark.benchmark.benchmark_analyzer import BenchmarkAnalyzer\n",
    "\n",
    "from bark.runtime.commons.parameters import ParameterServer\n",
    "\n",
    "from bark.runtime.viewer.matplotlib_viewer import MPViewer\n",
    "from bark.runtime.viewer.video_renderer import VideoRenderer\n",
    "\n",
    "\n",
    "from bark.core.models.behavior import BehaviorIDMClassic, BehaviorConstantAcceleration"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Database\n",
    "The benchmark database provides a reproducable set of scenarios.\n",
    "A scenario get's created by a ScenarioGenerator (we have a couple of them). The scenarios are serialized into binary files (ending `.bark_scenarios`) and packed together with the map file and the parameter files into a `.zip`-archive. We call this zipped archive a relase, which can be published at Github, or processed locally.\n",
    "\n",
    "## We will first start with the DatabaseSerializer\n",
    "\n",
    "The **DatabaseSerializer** recursively serializes all scenario param files sets\n",
    " within a folder.\n",
    " \n",
    "We will process the database directory from Github."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Do not set down the number of serialized scenarios in a release                     -> set serialized_scenarios=None\n",
      "INFO:root:The following list of files will be released:\n"
     ]
    },
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: '../../../benchmark_database/data/benchmark_database_tutorial.zip'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-5efd5ccfa9f3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mdbs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDatabaseSerializer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_scenarios\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_world_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_serialize_scenarios\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mdbs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"../../../benchmark_database/data/database1\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mlocal_release_filename\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdbs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mversion\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"tutorial\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Filename:'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocal_release_filename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.cache/bazel/_bazel_esterle/d337abac8c371120c1b9affa1049fa7e/execroot/bark_project/bazel-out/k8-fastbuild/bin/docs/tutorials/run.runfiles/benchmark_database/serialization/database_serializer.py\u001b[0m in \u001b[0;36mrelease\u001b[0;34m(self, version, github_token, delete)\u001b[0m\n\u001b[1;32m    137\u001b[0m         packed_file_name = os.path.join(os.path.dirname(self._database_dir),\"{}.{}\".format(\n\u001b[1;32m    138\u001b[0m             DatabaseSerializer._release_file_name(version), \"zip\"))\n\u001b[0;32m--> 139\u001b[0;31m         \u001b[0mDatabaseSerializer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_database_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpacked_file_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    140\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    141\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mgithub_token\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.cache/bazel/_bazel_esterle/d337abac8c371120c1b9affa1049fa7e/execroot/bark_project/bazel-out/k8-fastbuild/bin/docs/tutorials/run.runfiles/benchmark_database/serialization/database_serializer.py\u001b[0m in \u001b[0;36m_pack\u001b[0;34m(database_dir, packed_file_name)\u001b[0m\n\u001b[1;32m     99\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_pack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdatabase_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpacked_file_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m         \u001b[0mlogging\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'The following list of files will be released:'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m         \u001b[0mzipf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mzipfile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mZipFile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpacked_file_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'w'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mzipfile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mZIP_DEFLATED\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    102\u001b[0m         \u001b[0mtmp_dir\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"/tmp/bark_packed_databases/{}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muuid\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muuid4\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m         \u001b[0mshutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopytree\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdatabase_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtmp_dir\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# copy to resolve symlinks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.7/zipfile.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, file, mode, compression, allowZip64, compresslevel)\u001b[0m\n\u001b[1;32m   1205\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1206\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1207\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilemode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1208\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1209\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mfilemode\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmodeDict\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '../../../benchmark_database/data/benchmark_database_tutorial.zip'"
     ]
    }
   ],
   "source": [
    "dbs = DatabaseSerializer(test_scenarios=1, test_world_steps=5, num_serialize_scenarios=10)\n",
    "dbs.process(\"../../../benchmark_database/data/database1\")\n",
    "local_release_filename = dbs.release(version=\"tutorial\")\n",
    "\n",
    "print('Filename:', local_release_filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then reload to test correct parsing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:extracting zipped-database ../../../benchmark_database/data/benchmark_database_tutorial.zip to temporary directory /tmp/bark_extracted_databases/2c542f5e-4dbd-4779-821d-9f3f2c7aa328\n",
      "INFO:root:Found info dict set_info_40per_variation\n",
      "INFO:root:Found info dict set_info_20per_variation\n",
      "INFO:root:The following scenario sets are available\n",
      "INFO:root:\n",
      "                    GeneratorName  NumScenarios                                             Params    Seed                                         Serialized          SetName\n",
      "0  ConfigurableScenarioGeneration          10.0  scenario_sets/highway_merging/40per_variation....  2000.0  scenario_sets/highway_merging/40per_variation_...  40per_variation\n",
      "1  ConfigurableScenarioGeneration          10.0  scenario_sets/highway_merging/20per_variation....  2000.0  scenario_sets/highway_merging/20per_variation_...  20per_variation\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scenario:  <bark.runtime.scenario.scenario_generation.scenario_generation.ScenarioGeneration object at 0x7fe91b055890>\n",
      "Scenario:  <bark.runtime.scenario.scenario_generation.scenario_generation.ScenarioGeneration object at 0x7fe911324590>\n"
     ]
    }
   ],
   "source": [
    "db = BenchmarkDatabase(database_root=local_release_filename)\n",
    "scenario_generation, _ = db.get_scenario_generator(scenario_set_id=0)\n",
    "\n",
    "for scenario_generation, _ in db:\n",
    "  print('Scenario: ', scenario_generation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluators\n",
    "\n",
    "Evaluators allow to calculate a boolean, integer or real-valued metric based on the current simulation world state.\n",
    "\n",
    "The current evaluators available in BARK are:\n",
    "- StepCount: returns the step count the scenario is at.\n",
    "- GoalReached: checks if a controlled agent’s Goal Definitionis satisfied.\n",
    "- DrivableArea: checks whether the agent is inside its RoadCorridor.\n",
    "- Collision(ControlledAgent): checks whether any agent or only the currently controlled agent collided\n",
    "\n",
    "Let's now map those evaluators to some symbols, that are easier to interpret."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluators = {\"success\" : \"EvaluatorGoalReached\", \\\n",
    "              \"collision\" : \"EvaluatorCollisionEgoAgent\", \\\n",
    "              \"max_steps\": \"EvaluatorStepCount\"}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will now define the terminal conditions of our benchmark. We state that a scenario ends, if\n",
    "- a collision occured\n",
    "- the number of time steps exceeds the limit\n",
    "- the definition of success becomes true (which we defined to reaching the goal, using EvaluatorGoalReached)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "terminal_when = {\"collision\" :lambda x: x, \\\n",
    "                 \"max_steps\": lambda x : x>10, \\\n",
    "                 \"success\" : lambda x: x}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Behaviors Under Test\n",
    "Let's now define the Behaviors we want to compare. We will compare IDM with Constant Velocity, but we could also compare two different parameter sets for IDM. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = ParameterServer() \n",
    "behaviors_tested = {\"IDM\": BehaviorIDMClassic(params), \"Const\" : BehaviorConstantAcceleration(params)}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Benchmark Runner\n",
    "\n",
    "The BenchmarkRunner allows to evaluate behavior models with different parameter configurations over the entire benchmarking database."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:BenchmarkRunner:Running config idx 0/39: Scenario 0 of set \"40per_variation\" for behavior \"IDM\"\n",
      "INFO:BenchmarkRunner:Running config idx 1/39: Scenario 1 of set \"40per_variation\" for behavior \"IDM\"\n",
      "INFO:BenchmarkRunner:Running config idx 2/39: Scenario 2 of set \"40per_variation\" for behavior \"IDM\"\n",
      "INFO:BenchmarkRunner:Running config idx 3/39: Scenario 3 of set \"40per_variation\" for behavior \"IDM\"\n",
      "INFO:BenchmarkRunner:Running config idx 4/39: Scenario 4 of set \"40per_variation\" for behavior \"IDM\"\n",
      "INFO:BenchmarkRunner:Running config idx 5/39: Scenario 5 of set \"40per_variation\" for behavior \"IDM\"\n",
      "INFO:BenchmarkRunner:Running config idx 6/39: Scenario 6 of set \"40per_variation\" for behavior \"IDM\"\n",
      "INFO:BenchmarkRunner:Running config idx 7/39: Scenario 7 of set \"40per_variation\" for behavior \"IDM\"\n",
      "INFO:BenchmarkRunner:Running config idx 8/39: Scenario 8 of set \"40per_variation\" for behavior \"IDM\"\n",
      "INFO:BenchmarkRunner:Running config idx 9/39: Scenario 9 of set \"40per_variation\" for behavior \"IDM\"\n",
      "INFO:BenchmarkRunner:\n",
      "------------------- Current Evaluation Results ---------------------- \n",
      " Num. Results:10\n",
      "                           success  collision  max_steps  step\n",
      "scen_set        behavior                                     \n",
      "40per_variation IDM         False      False       11.0  10.0 \n",
      " ---------------------------------------------------------------------\n",
      "INFO:BenchmarkRunner:Running config idx 10/39: Scenario 0 of set \"20per_variation\" for behavior \"IDM\"\n",
      "INFO:BenchmarkRunner:Running config idx 11/39: Scenario 1 of set \"20per_variation\" for behavior \"IDM\"\n",
      "INFO:BenchmarkRunner:Running config idx 12/39: Scenario 2 of set \"20per_variation\" for behavior \"IDM\"\n",
      "INFO:BenchmarkRunner:Running config idx 13/39: Scenario 3 of set \"20per_variation\" for behavior \"IDM\"\n",
      "INFO:BenchmarkRunner:Running config idx 14/39: Scenario 4 of set \"20per_variation\" for behavior \"IDM\"\n",
      "INFO:BenchmarkRunner:Running config idx 15/39: Scenario 5 of set \"20per_variation\" for behavior \"IDM\"\n",
      "INFO:BenchmarkRunner:Running config idx 16/39: Scenario 6 of set \"20per_variation\" for behavior \"IDM\"\n",
      "INFO:BenchmarkRunner:Running config idx 17/39: Scenario 7 of set \"20per_variation\" for behavior \"IDM\"\n",
      "INFO:BenchmarkRunner:Running config idx 18/39: Scenario 8 of set \"20per_variation\" for behavior \"IDM\"\n",
      "INFO:BenchmarkRunner:Running config idx 19/39: Scenario 9 of set \"20per_variation\" for behavior \"IDM\"\n",
      "INFO:BenchmarkRunner:\n",
      "------------------- Current Evaluation Results ---------------------- \n",
      " Num. Results:20\n",
      "                           success  collision  max_steps  step\n",
      "scen_set        behavior                                     \n",
      "20per_variation IDM         False      False       11.0  10.0\n",
      "40per_variation IDM         False      False       11.0  10.0 \n",
      " ---------------------------------------------------------------------\n",
      "INFO:BenchmarkRunner:Running config idx 20/39: Scenario 0 of set \"40per_variation\" for behavior \"Const\"\n",
      "INFO:BenchmarkRunner:Running config idx 21/39: Scenario 1 of set \"40per_variation\" for behavior \"Const\"\n",
      "INFO:BenchmarkRunner:Running config idx 22/39: Scenario 2 of set \"40per_variation\" for behavior \"Const\"\n",
      "INFO:BenchmarkRunner:Running config idx 23/39: Scenario 3 of set \"40per_variation\" for behavior \"Const\"\n",
      "INFO:BenchmarkRunner:Running config idx 24/39: Scenario 4 of set \"40per_variation\" for behavior \"Const\"\n",
      "INFO:BenchmarkRunner:Running config idx 25/39: Scenario 5 of set \"40per_variation\" for behavior \"Const\"\n",
      "INFO:BenchmarkRunner:Running config idx 26/39: Scenario 6 of set \"40per_variation\" for behavior \"Const\"\n",
      "INFO:BenchmarkRunner:Running config idx 27/39: Scenario 7 of set \"40per_variation\" for behavior \"Const\"\n",
      "INFO:BenchmarkRunner:Running config idx 28/39: Scenario 8 of set \"40per_variation\" for behavior \"Const\"\n",
      "INFO:BenchmarkRunner:Running config idx 29/39: Scenario 9 of set \"40per_variation\" for behavior \"Const\"\n",
      "INFO:BenchmarkRunner:\n",
      "------------------- Current Evaluation Results ---------------------- \n",
      " Num. Results:30\n",
      "                           success  collision  max_steps  step\n",
      "scen_set        behavior                                     \n",
      "20per_variation IDM         False      False       11.0  10.0\n",
      "40per_variation Const       False      False       11.0  10.0\n",
      "                IDM         False      False       11.0  10.0 \n",
      " ---------------------------------------------------------------------\n",
      "INFO:BenchmarkRunner:Running config idx 30/39: Scenario 0 of set \"20per_variation\" for behavior \"Const\"\n",
      "INFO:BenchmarkRunner:Running config idx 31/39: Scenario 1 of set \"20per_variation\" for behavior \"Const\"\n",
      "INFO:BenchmarkRunner:Running config idx 32/39: Scenario 2 of set \"20per_variation\" for behavior \"Const\"\n",
      "INFO:BenchmarkRunner:Running config idx 33/39: Scenario 3 of set \"20per_variation\" for behavior \"Const\"\n",
      "INFO:BenchmarkRunner:Running config idx 34/39: Scenario 4 of set \"20per_variation\" for behavior \"Const\"\n",
      "INFO:BenchmarkRunner:Running config idx 35/39: Scenario 5 of set \"20per_variation\" for behavior \"Const\"\n",
      "INFO:BenchmarkRunner:Running config idx 36/39: Scenario 6 of set \"20per_variation\" for behavior \"Const\"\n",
      "INFO:BenchmarkRunner:Running config idx 37/39: Scenario 7 of set \"20per_variation\" for behavior \"Const\"\n",
      "INFO:BenchmarkRunner:Running config idx 38/39: Scenario 8 of set \"20per_variation\" for behavior \"Const\"\n",
      "INFO:BenchmarkRunner:Running config idx 39/39: Scenario 9 of set \"20per_variation\" for behavior \"Const\"\n",
      "INFO:BenchmarkRunner:\n",
      "------------------- Current Evaluation Results ---------------------- \n",
      " Num. Results:40\n",
      "                           success  collision  max_steps  step\n",
      "scen_set        behavior                                     \n",
      "20per_variation Const       False      False       11.0  10.0\n",
      "                IDM         False      False       11.0  10.0\n",
      "40per_variation Const       False      False       11.0  10.0\n",
      "                IDM         False      False       11.0  10.0 \n",
      " ---------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "benchmark_runner = BenchmarkRunner(benchmark_database=db,\\\n",
    "                                   evaluators=evaluators,\\\n",
    "                                   terminal_when=terminal_when,\\\n",
    "                                   behaviors=behaviors_tested,\\\n",
    "                                   log_eval_avg_every=10)\n",
    "\n",
    "result = benchmark_runner.run(maintain_history=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will now dump the files, to allow them to be postprocessed later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Saved BenchmarkResult to /tmp/bark_extracted_databases/2c542f5e-4dbd-4779-821d-9f3f2c7aa328/benchmark_results.pickle\n"
     ]
    }
   ],
   "source": [
    "result.dump(os.path.join(\"./benchmark_results.pickle\"))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Benchmark Results\n",
    "\n",
    "Benchmark results contain\n",
    "- the evaluated metrics of each simulation run, as a Panda Dataframe\n",
    "- the world state of every simulation (optional)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_loaded = BenchmarkResult.load(os.path.join(\"./benchmark_results.pickle\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will now first analyze the dataframe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Terminal</th>\n",
       "      <th>behavior</th>\n",
       "      <th>collision</th>\n",
       "      <th>config_idx</th>\n",
       "      <th>max_steps</th>\n",
       "      <th>scen_idx</th>\n",
       "      <th>scen_set</th>\n",
       "      <th>step</th>\n",
       "      <th>success</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[max_steps]</td>\n",
       "      <td>IDM</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>40per_variation</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[max_steps]</td>\n",
       "      <td>IDM</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>40per_variation</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[max_steps]</td>\n",
       "      <td>IDM</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>40per_variation</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[max_steps]</td>\n",
       "      <td>IDM</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>40per_variation</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[max_steps]</td>\n",
       "      <td>IDM</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>40per_variation</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Terminal behavior  collision  config_idx  max_steps  scen_idx  \\\n",
       "0  [max_steps]      IDM      False           0         11         0   \n",
       "1  [max_steps]      IDM      False           1         11         1   \n",
       "2  [max_steps]      IDM      False           2         11         2   \n",
       "3  [max_steps]      IDM      False           3         11         3   \n",
       "4  [max_steps]      IDM      False           4         11         4   \n",
       "\n",
       "          scen_set  step  success  \n",
       "0  40per_variation    10    False  \n",
       "1  40per_variation    10    False  \n",
       "2  40per_variation    10    False  \n",
       "3  40per_variation    10    False  \n",
       "4  40per_variation    10    False  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = result_loaded.get_data_frame()\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Benchmark Analyzer\n",
    "\n",
    "The benchmark analyzer allows to filter the results to show visualize what really happened. These filters can be set via a dictionary with lambda functions specifying the evaluation criteria which must be fullfilled.\n",
    "\n",
    "A config is basically a simulation run, where step size, controlled agent, terminal conditions and metrics have been defined.\n",
    "\n",
    "Let us first load the results into the BenchmarkAnalyzer and then filter the results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyzer = BenchmarkAnalyzer(benchmark_result=result_loaded)\n",
    "\n",
    "\n",
    "configs_idm = analyzer.find_configs(criteria={\"behavior\": lambda x: x==\"IDM\", \"success\": lambda x : not x})\n",
    "configs_const = analyzer.find_configs(criteria={\"behavior\": lambda x: x==\"Const\", \"success\": lambda x : not x})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will now create a video from them. We will use Matplotlib Viewer and render everything to a video."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjwAAAJJCAYAAACu4O3oAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGxhJREFUeJzt3X+M5Hd93/H3Z3b3fu7d7vk3DtTG8OXrM9TnQhBN6rRpq9IGQYAKiRJoUFVBK6WNUjegoiIIbZOGSkkJDaWtCm2RgxL6I1FC0yRtQOAklIgD0RabL19cOBWE8Z3B99P23e58+8eO4bg7f31735v72u99PCTE3sx8Z14zt9p57syur3RdFwAAmU3GHgAAMG+CBwBIT/AAAOkJHgAgPcEDAKQneACA9AQPAJCe4OFpp5TyjFLKPyml/L1Syk9cxOV/tpTyulLKa5/g/Pec9fHNpZSfuojrfGsp5d+fc9r3HFtK+eFSygdKKW8vpfz9C1zHz5RSVp/stmaX/dFSys6Luezs8u+Z/f9vlVJ+spTyj0spt882/dHsvNeUUn7zQsfNPq5LKe8tpdxVSnnxxd72k+z6wVLKs845bamU8r5Syk+XUv7qBY55wseplPLcUsqnSymrpZQXzj4vfrGUsvty7H0iF/pcOusxr0spd2ziuq74Y37u7V7oz09wXW8upfzG5dgFV9ri2APgEvxERPx813UnIiJKKS+IiB+LiC4iPhwRfzci7omIvxgR74yIF0fExyLiFaWUj0TEP4+I+yLiZV3XvXJ2HbdHxF+PiOMRcayU8qqI2B4RfyYi/mFEvKHruvc/PqDrun92MU8QM1dFxGdKKdfMrutrEfHl2Xk/Pntiu2u2s46IbRHxhxFxQ0T8fkT81Oy+fbKU8tMR8a2IWI2IX4mIX4iIfxcRpyPiy13X/d9zbvv+ruveW0qZzO73r0fE50opfzYiDkTEuZc/29Wz2/3jiDhYSvmhiPjzEfHg7Db/aUQcmj1m04h41uy+/tuI2BcRL4qIZ3Rd9w9KKZ+IiH8ZETsj4lQp5ZWx8Q3XjRHxs7Pb+q8R8YkLPE5VRPxYKeXDEfG6x/8eSikrEfHqiPhvs72vi4i3RcQPRsRfKqW8ISL+S0TcHhFvj4h3RcQ3Y+Pr3mcj4rUR8R+7rvvY7PpKRPxC13V3lVJ+cXb/XhcRt0TEL0fEW2Z/L38QET9QSvl0RLwiIm6NiJ+PiNtKKX8tIh6IiNVSym0R8YzZfXx7RPxaRPyniNjfdd3bRn7M3za7z49//MXZn18SG5+Hz4yIfxERP3rO5/2/md0veNrxCg9PRyU2nhQe97LY+OL8y7OPH+u67lci4n/PLntv13W/P7vsdRFxrOu6fx0RR8+6jh+JjSeHj0REdF33GxHxNyPiY13XHT/7i/4m/VbXdY/HzA9HxGMR8VBsPKFERNwdGxGyPyL2zDa9sOu6P4iIOyPiDRHxobOu76qu694XG0+MERF/2HXdR7uu+70LxM65Hn/M/kdEvCkivtB74a77o4j4mYi4JjaeaF/ddd0/6rruX0XEn4yIvbO9184O+ehs64tm9+Xk7HIREZ/vuu7Xzrr653Rd997Z8Tsi4o2zjz8Q5z9ObUR8uOu6h8/5e/jLsfE17E9HxEvPuY8RESe7rvtwRHw1Il4YEc+PiGOxEQYREf/98diZ3d8uIv5fKeX7Z7e9NDvrSET8QGx8Xn3orMd5V2xEx+mIuDk2Ps9+9azbf3HXdb8QEZ+JiBdExFe6rvtQbATIBV3Bx/ya2Wk/FBtR+IHZ7X86NgLxga7rvj7g8x6ecgQPT0fvi4i3nfWW1m/Hxqs6fyc2XiV4/Emvi/M/xx+MiJVSyptj4wnicb8TEa+Pje/Yo5Typth45eHPlVKuOfets1LKj0fEnyql/MiTbH3F7O2s+yPiE7PTdkTE/5p9PJ39bxIb0fNYfPeJ9ksRcUfXdfefdX3fmm156Kzjo5TyV0opt1zg9p9TSvnJ2Hh144Oz07rYeFL9z33DSykHYiP6DsTGKwC/Xkp5x+yx+T+xEQI7Zh8/fr2PP+YviIhTZ92X6TlXf/9s10psBMNdsfHk++U4/3FqIuJvl1L2nf330HXdR7que3dE/M+I+L2I+NXZ/XplbETdzlLK34qN0Pns7Lp2xndD79xNERvxcHdE/IfYiJiF2f1ZuMDlb4uIR2fnLUTE0VLK3zjr/D+e/d1//+wxetJ/x+cKPuZHZqfdExux+NrZ7b88Nh67lVJKdYHP+1fFxuf9Bd8ehqey4t/SYqsppbwsIp4dETF7teRyXe/NEfGqrusu9q2up5xSynu6rnvSn2F6Oni63Jeny054uvMzPGw5Xdf99pyu+uHYeLXh6ex3xh5wuVxMRJRS/kRE/IXZH7/Vdd1v9l1+TtI85vBU5hUeACA9P8MDAKQneACA9AQPAJCe4AEA0hM8AEB6ggcASE/wAADpCR4AID3BAwCkJ3gAgPQEDwCQnuABANITPABAeoIHAEhP8AAA6QkeACA9wQMApCd4AID0BA8AkN5i35nXXHNNd/PNN1+hKcA8HD16NFZWVr7ntLW1tTh16lR0Xbep61pYWIjl5eXLOe+iXOg+AJzr4MGDR7quu/ZC5/UGz8033xyf+cxn5rMKuCI++tGPxstf/vLvOe3w4cNx8ODBWFtb29R1ra6uxp133nk5512UC90HgHOVUg490Xne0gIA0hM8AEB6ggcASE/wAADpCR4AID3BAwzWNM3YEwB6CR5gsLZtx54A0EvwAADpCR4AID3BAwCkJ3iAwaqqGnsCQC/BA1vQ4mLvP6O36ePquh4yB2DuBA9sQbt27YrpdLrp4/bu3TuHNQDzJ3hgC9q2bdumj1lYWIjdu3fPYQ3A/Ake2IJKKbFz585NH7e8vDyHNQDzJ3hgi7r11ltjYWHhoi+/e/fuuOqqq+a4CGB+BA9sUTfccMNF/0zOZDKJAwcORCllzqsA5kPwwBZVSonbb789JpMn+TKwthY33nhjrKysXJlhAHMgeGAL27NnT9xxxx2x7+DB2HHkSJS1tZicORNlfT0WT56MPV/9atzw8Y/HbbfdNvZUgEEu7T/GAaRx4403xuIHPxiTa6+NrpQ4vXdvLJ46FQtnzkRExPTw4dj2S7808kqAYbzCA3xH6brYfvTod2LnYjVNM6dFAJeH4AEGa9t27AkAvQQPAJCe4AEA0hM8AEB6ggcYrKqqsScA9BI8wGB1XY89AaCX4AEA0hM8AEB6ggcASE/wAADpCR4AID3BAwCkJ3gAgPQWxx4A5PfQQw/FN7/5zTh69GicOHEi1tfXY9euXbFnz55YXV2Nm266KSYT338B8yN4gMGapnnC//jgV77ylbjvvvtiOp1+z+nHjh2LY8eOxTe+8Y34+te/Hi95yUtiaWnpSswFtiDfUgGDtW173mld18W9994bX/ziF8+LnbNNp9M4duxY3HPPPfHII4/McyawhQkeYC4eeOCBOHToUKyvrz/pZafTaZw6dSo++9nPXoFlwFYkeIC5OH78+EXFztlOnDgxpzXAVid4gLk4fvz4po9ZW1vrffsL4FIJHmCwqqrOO+3kyZObvp7JZBKnTp26HJMAvofgAQa70G9ora2tbfp6SimXdBzAkxE8AEB6ggcASE/wAADpCR4AID3BAwCkJ3gAgPQEDwCQnuABANITPMBgTdOMPQGgl+ABBmvbduwJAL0EDwCQnuABANITPABAeoIHGKyqqrEnAPQSPMBgdV2PPQGgl+ABANITPABAeoIHAEhP8ABPKaWUsScACQkeYC527Nix6WOm02ls3759DmuArU7wAHOxvLy86WO6rhM8wFwIHmAu9uzZE5PJ5r7EbN++3VtawFwIHmAu9u3bt6l4KaXE1VdfPcdFwFYmeIDBmqY577TV1dW49tprLzp6JpNJ7N+//3JPA4gIwQNcBm3bXvD05z//+Rf1ttbCwkJUVXVJP+gMcDEEDzA3O3fujOc85zmxsLAQCwsL550/mUxicXExlpaW4pZbbhlhIbBVLI49AMitqqq47rrr4vjx43H06NE4evRonDlzJvbu3RsrKyuxd+/eWF1d3fQPOANshuAB5qqUEqurq7G6uhrPetazxp4DbFG+pQIGq6pq7AkAvQQPMFhd12NPAOgleACA9AQPAJCe4AEA0hM8AEB6ggcASE/wAADpCR4AID3BAwCkJ3iAwZqmGXsCQC/BAwzWtu3YEwB6CR4AID3BAwCkJ3gAgPQEDzBYVVVjTwDoJXiAweq6HnsCQC/BAwCkJ3gAgPQEDwCQnuABANITPABAeoIHAEhP8AAA6QkeACA9wQMM1jTN2BMAegkeYLC2bceeANBL8AAA6QkeACA9wQMApCd4gMGqqhp7AkAvwQMMVtf12BMAegkeACA9wQMApCd4AID0BA8AkJ7gAQDSEzwAQHqCBwBIT/AAAOkJHmCwpmnGngDQS/AAg7VtO/YEgF6CBwBIT/AAAOkJHgAgPcEDDFZV1dgTAHoJHmCwuq7HngDQS/AAAOkJHgAgPcEDAKQneACA9AQPAJCe4AEA0hM8AEB6ggcASE/wAIM1TTP2BIBeggcYrG3bsScA9BI8AEB6ggcASE/wAADpCR5gsKqqxp4A0EvwAIPVdT32BIBeggcASE/wAADpCR4AID3BAwCkJ3gAgPQEDwCQnuABANITPABAeoIHGKxpmrEnAPQSPMBgbduOPQGgl+ABANITPABAeoIHAEhP8ACDVVU19gSAXoIHGKyu67EnAPQSPABAeoIHAEhP8AAA6QkeACA9wQMApCd4AID0BA8AkJ7gAQDSEzzAYE3TjD0BoJfgAQZr23bsCQC9BA8AkJ7gAQDSEzwAQHqCBxisqqqxJwD0EjzAYHVdjz0BoJfgAQDSEzwAQHqCBwBIT/AAAOkJHgAgPcEDAKQneACA9AQPAJCe4AEGa5pm7AkAvQQPMFjbtmNPAOgleACA9AQPAJCe4AEA0hM8wGBVVY09AaCX4AEGq+t67AkAvQQPAJCe4AEA0hM8AEB6ggcASE/wAADpCR4AID3BAwCkJ3gAgPQEDzBY0zRjTwDoJXiAwdq2HXsCQC/BAwCkJ3gAgPQEDwCQnuABBquqauwJAL0EDzBYXddjTwDoJXgAgPQEDwCQnuABANITPABAeoIHAEhP8AAA6QkeACA9wQMApCd4gMGaphl7AkAvwQMM1rbt2BMAegkeACA9wQMApCd4AID0BA8wWFVVY08A6LU49gBgfI9dfXU8+rznxSPXXBOnrr8+lk6ciF0PPhg7jxyJHd/+9pMeX9f1FVgJcOkED2xhXdfFfffdF195+9tjMp3G+uJixOJiRNfF5PTpiK6LxVOnYu+pU7Fr166x5wJcMm9pwRY1nU7j4MGDcejQoei2bYv1HTs2YiciopSYbt8e0x074vTqatxzzz3x8MMPjzsYYADBA1vUwYMH48EHH4z19fX+C04mcebMmfjUpz4VJ06cuDLjAC4zwQNb1JEjR2I6nW7qGK/yAE9Xgge2oLW1tU3Hzvr6epw8eXJOiwDmS/DAFnTq1KlYWFjY9HHHjx+fwxqA+RM8sAU99thjl3Tco48+epmXAFwZggcASE/wAIM1TTP2BIBeggcYrG3bsScA9BI8AEB6ggcASE/wAADpCR5gsKqqxp4A0EvwAIPVdT32BIBeggcASE/wAADpCR4AID3BAwCkJ3gAgPQEDwCQnuABANITPABAeoIHGKxpmrEnAPQSPMBgbduOPQGgl+ABANITPLAFLS4uXtJxS0tLl3kJwJUheGAL2rVrV0yn000ft2fPnjmsAZg/wQNb0LZt2zZ9zMLCQiwvL1/wvKqqhk4CmCvBA1tQKSV27dq16eOeKHjquh46CWCuBA9sUbfeemssLCxc9OWXl5dj3759c1wEMD+CB7ao66+/PlZWVi7qspPJJA4cOBCllDmvApgPwQNbVCklbr/99phM+r8MlLW1eOYznxl79+69QssALj/BA1vY8vJyvOhFL4prPvWp2PnAAzE5fTrK2lrE+npsO3o0Vr785XjG7/5u7N+/f+ypAINc2n+MA0jj+uuvj3L33TG59troImJt9+5YePTRmKyvR0TE9PDhWHr/+8cdCTCQ4AG+o0TE0smTY88AuOwEDzB305MnY71pYu2rX421ponu4Ydjcf/+WHj2s2Pxuc+Nhe/7vrEnAskJHmCu1to2Hn7962N67FjE+nrE2lrEZBJd10XZti1ifT12v+UtsetNb/JbYMDc+KFlYLCmaS54+mOf/nR8+9WvjunDD8dkeTkmKysxufrqmOzbFwtXXRWT5eUou3bFyXe/O068853RzX5uCOByEzzAYG3bnnfamXvvjaNvfGPEdBqTnl9pL0tLUfbujVN33x0n3vWuec4EtjDBA8zFmc99LuL06SgX8U9YlIWFmOzeHY99/ONXYBmwFQkeYC7W779/cwcsLcX0G9+I7hL+FXeAJyN4gLlY+9KXIpaWLvryZTKJKCWmR47McRWwVQkeYLCqqs47bXr4cJTFTf4i6MJCTB966DKtAvguwQMMVtf15bkiv5YOzIngAQDSEzwAQHqCBwBIT/AAAOkJHgAgPcEDAKQneACA9AQPAJCe4AEGa5pm7AkAvQQPMFjbtmNPAOgleACA9AQPAJCe4AEA0hM8wGBVVY09AaCX4AEGq+t67AkAvQQPAJCe4AEA0hM8wFNH1429AEhqcewBwPgmN90U00OHnvC8S7rOlZVYP3QoymYOmk5jsmfPJd0eQB/BA8Q1n/zkZb/Ohec+N84cPHjRl++m04j19ZjccMNl3wLgLS1gLhaf97zoNvMW1ZkzMbnuuiiLvg8DLj/BA8zFwi23RFlYuOjo6R55JBb375/zKmCrEjzAXGy7886NV3mOHXvSy3br6xERsfuuu+Y9C9iiBA8wWNM0551WJpPY83M/F9F1Gz+f06M7dix2vOY1sfSCF8xrIrDFCR5gsLZtL3j60oEDsf2Vr4zp4cMxPXo0ukcfjW46ja7rojt9OqYnTsT0yJEoO3fG8lvfeoVXA1uJnw4E5mrPO94Ri3Uda5//fKx94Qux/rWvRUynUfbti6XbboulO+6I7S99aUyuumrsqUBiggeYq8nKSux+85u/8+dubS26xx6Lye7dI64CthrBA1xRZXHRr54DV5yf4QEGq6pq7AkAvQQPMFhd12NPAOgleACA9AQPAJCe4AEA0hM8AEB6ggcASE/wAADpCR4AID3BAwCkJ3iAwZqmGXsCQC/BAwzWtu3YEwB6CR4AID3BAwCkJ3gAgPQEDzBYVVVjTwDoJXiAweq6HnsCQC/BAwCkJ3gAgPQEDwCQnuABANITPABAeoIHAEhP8AAA6QkeACA9wQMM1jTN2BMAegkeYLC2bceeANBL8AAA6QkeACA9wQMApCd4gMGqqhp7AkAvwQMMVtf12BMAegkeACA9wQMApCd4AID0BA8AkJ7gAQDSEzwAQHqCBwBIT/AAAOkJHmCwpmnGngDQS/AAg7VtO/YEgF6CBwBIT/AAAOkJHgAgPcEDDFZV1dgTAHoJHmCwuq7HngDQS/AAAOkJHgAgPcEDAKQneACA9AQPAJCe4AEA0hM8AEB6ggcASE/wAIM1TTP2BIBeggcYrG3bsScA9BI8AEB6ggcASE/wAADpCR5gsKqqxp4A0EvwAIPVdT32BIBeggcASE/wAADpCR4AID3BAwCkJ3gAgPQEDwCQnuABANITPABAeoIHGKxpmrEnAPQSPMBgbduOPQGgl+ABANITPABAeoIHAEhP8ACDVVU19gSAXoIHGKyu67EnAPQSPABAeoIHAEhP8AAA6QkeACA9wQMApCd4AID0BA8AkJ7gAQDSEzzAYE3TjD0BoJfgAQZr23bsCQC9BA8AkJ7gAQDSEzwAQHqCBxisqqqxJwD0EjzAYHVdjz0BoJfgAQDSEzwAQHqCBwBIT/AAAOkJHgAgPcEDAKQneACA9AQPAJCe4AEGa5pm7AkAvQQPMFjbtmNPAOgleACA9AQPAJCe4AEA0hM8wGBVVY09AaCX4AEGq+t67AkAvQQPAJCe4AEA0hM8AEB6ggcASE/wAADpCR4AID3BAwCkJ3gAgPQEDzBY0zRjTwDoJXiAwdq2HXsCQC/BAwCkJ3gAgPQEDwCQnuABBquqauwJAL0EDzBYXddjTwDoJXgAgPQEDwCQnuABANITPABAeoIHAEhP8AAA6QkeACA9wQMApCd4gMGaphl7AkAvwQMM1rbt2BMAegkeACA9wQMApCd4AID0BA8wWFVVY08A6CV4gMHquh57AkAvwQMApCd4AID0BA8AkJ7gAQDSEzwAQHqCBwBIT/AAAOkJHgAgPcEDDNY0zdgTAHoJHmCwtm3HngDQS/AAAOkJHgAgPcEDAKQneIDBqqoaewJAL8EDDFbX9dgTAHoJHgAgPcEDAKQneACA9AQPAJCe4AEA0hM8AEB6ggcASE/wAADpCR5gsKZpxp4A0EvwAIO1bTv2BIBeggcASE/wAADpCR4AID3BAwxWVdXYEwB6CR5gsLqux54A0EvwAADpCR4AID3BAwCkJ3gAgPQEDwCQnuABANITPABAeoIHAEhP8ACDNU0z9gSAXoIHGKxt27EnAPQSPABAeoIHAEhP8AAA6QkeYLCqqsaeANBL8ACD1XU99gSAXoIHAEhP8AAA6QkeACA9wQMApCd4AID0BA8AkJ7gAQDSEzwAQHqCBxisaZqxJwD0EjzAYG3bjj0BoJfgAQDSEzwAQHqCBwBIT/AAg1VVNfYEgF6CBxisruuxJwD0EjwAQHqCBwBIT/AAAOkJHgAgPcEDAKQneACA9AQPAJCe4AEA0hM8wGBN04w9AaCX4AEGa9t27AkAvQQPAJCe4AEA0hM8AEB6ggcYrKqqsScA9BI8wGB1XY89AaCX4AEA0hM8AEB6ggcASE/wAADpCR4AID3BAwCkJ3gAgPQEDwCQnuABBmuaZuwJAL0EDzBY27ZjTwDoJXgAgPQEDwCQnuABANITPMBgVVWNPQGgl+ABBqvreuwJAL0EDwCQnuABANITPABAeoIHAEhP8AAA6QkeACA9wQMApCd4AID0BA8wWNM0Y08A6CV4gMHath17AkAvwQMApCd4AID0BA8AkJ7gAQarqmrsCQC9BA8wWF3XY08A6CV4AID0BA8AkJ7gAQDSEzwAQHqCBwBIT/AAAOkJHgAgPcEDAKQneIDBmqYZewJAL8EDDNa27dgTAHoJHgAgPcEDAKQneACA9AQPMFhVVWNPAOgleIDB6roeewJAL8EDAKQneACA9AQPAJCe4AEA0hM8AEB6peu6Jz6zlMMRcejKzQEAuGQ3dV137YXO6A0eAIAMvKUFAKQneACA9AQPAJCe4AEA0hM8AEB6/x+EhtXszGn6pgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sim_step_time=0.2\n",
    "\n",
    "params2 = ParameterServer()\n",
    "\n",
    "fig = plt.figure(figsize=[10, 10])\n",
    "viewer = MPViewer(params=params2, y_length = 80, enforce_y_length=True, enforce_x_length=False,\\\n",
    "                  follow_agent_id=True, axis=fig.gca())\n",
    "video_exporter = VideoRenderer(renderer=viewer, world_step_time=sim_step_time)\n",
    "\n",
    "analyzer.visualize(viewer = video_exporter, real_time_factor = 1, configs_idx_list=configs_idm[1:3], \\\n",
    "                  fontsize=6)\n",
    "                   \n",
    "video_exporter.export_video(filename=\"/tmp/tutorial_video\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
